package com.crawler;

import com.crawler.core.HttpClientDownloader;
import com.crawler.pageProcessor.CnBlogPageProcessor;
import us.codecraft.webmagic.Spider;

public class Main {
    public static void main(String[] args) throws Exception {


//        Spider.create(new GithubRepoPageProcessor())
//                // 从"https://github.com/code4craft"开始抓
//                .addUrl("https://github.com/code4craft")
//                // 开启5个线程抓取
//                .thread(5)
//                // 启动爬虫
//                .run();

//         启动爬虫
//        Spider.create(new DouBanReadPageProcessor())
//                .setDownloader(new HttpClientDownloader())
//                .addUrl("https://book.douban.com/tag/小说")  // 豆瓣图书小说标签页面
//                .run();

        Spider.create(new CnBlogPageProcessor())
            .setDownloader(new HttpClientDownloader())
            .addUrl("https://www.cnblogs.com/#")
            .run();

    }

}